{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# BERT + SAN(Stochastic Answer Networks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from functools import partial\n",
    "from itertools import tee\n",
    "from collections import namedtuple\n",
    "import os\n",
    "from datetime import datetime\n",
    "\n",
    "import pandas as pd\n",
    "import torch\n",
    "from torch import nn\n",
    "from torch import optim\n",
    "from torch.nn import functional as F\n",
    "from torch.utils.data import Dataset, DataLoader, Subset\n",
    "from tqdm import tqdm_notebook\n",
    "\n",
    "from transformers import BertTokenizer, BertModel\n",
    "\n",
    "device = torch.device('cuda')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Loading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SANDataLoading(Dataset):\n",
    "    def __init__(self, ids, passages, options, answers, pretrained_model='bert-base-cased'):\n",
    "        \n",
    "        self.tokenizer = BertTokenizer.from_pretrained(pretrained_model, \n",
    "                                                       do_basic_tokenize=True,\n",
    "                                                       never_split=['<BLANK>'])\n",
    "        \n",
    "        self.MAX_SEQ_LEN = 512\n",
    "        \n",
    "        self.ids = ids\n",
    "        self.passages = passages\n",
    "        self.options = options\n",
    "        self.answers = answers\n",
    "        \n",
    "    @classmethod    \n",
    "    def from_tsv(cls, data_path, pretrained_model='bert-base-cased'):\n",
    "        dataframe = pd.read_csv(data_path, sep='\\t', index_col=0)\n",
    "        \n",
    "        ids = dataframe.index.tolist()\n",
    "        passages = dataframe['본문'].tolist()\n",
    "        options = list(zip(dataframe['보기1'],dataframe['보기2'],dataframe['보기3'],\n",
    "                           dataframe['보기4'],dataframe['보기5']))\n",
    "        answers = dataframe['정답'].tolist()\n",
    "        return cls(ids, passages, options, answers, pretrained_model)\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self._ids)\n",
    "    \n",
    "    def __getitem__(self,idx):\n",
    "        \n",
    "        id_ = self._ids[idx]\n",
    "        passage_input_id = self._passage_input_ids[idx]\n",
    "        passage_token_type_id = self._passage_token_type_ids[idx]\n",
    "        passage_attention_mask = self._passage_attention_masks[idx] \n",
    "        option_input_id = self._option_input_ids[idx]\n",
    "        option_token_type_id = self._option_token_type_ids[idx]\n",
    "        option_attention_mask = self._option_attention_masks[idx]\n",
    "        answer = self._answers[idx]\n",
    "        \n",
    "        return id_, passage_input_id, passage_token_type_id, passage_attention_mask, \\\n",
    "    option_input_id, option_token_type_id, option_attention_mask, answer\n",
    "    \n",
    "    def _get_tokens(self, text, is_passage):\n",
    "        \n",
    "        first_end_idx = text.find('<BLANK>')\n",
    "        last_start_idx = first_end_idx + len('<BLANK>')\n",
    "        if is_passage:\n",
    "            first = self.tokenizer.tokenize(text[:first_end_idx])\n",
    "            last = self.tokenizer.tokenize(text[last_start_idx:])\n",
    "            tokens = first + ['<BLANK>'] + last\n",
    "        else:\n",
    "            tokens = self.tokenizer.tokenize(text)\n",
    "        return tokens\n",
    "    \n",
    "    def _add_special_token(self, tokens, is_passage):\n",
    "        \n",
    "        temp = None\n",
    "        if is_passage:\n",
    "            blank_idx = next(i for i, token in enumerate(tokens) if token =='<BLANK>')\n",
    "        \n",
    "            first = ['[CLS]'] + tokens[:blank_idx] + ['[SEP]']\n",
    "            last = tokens[blank_idx+1:] + ['[SEP]']\n",
    "            temp = first+last\n",
    "            \n",
    "        else:\n",
    "            temp = ['[CLS]'] + tokens + ['[SEP]']\n",
    "            \n",
    "        pad = ['[PAD]'] * (self.MAX_SEQ_LEN - len(temp))\n",
    "            \n",
    "        return temp + pad\n",
    "    \n",
    "    def _get_input_ids(self, tokens):\n",
    "        return self.tokenizer.convert_tokens_to_ids(tokens)\n",
    "    \n",
    "    def _get_token_type_ids(self, tokens):\n",
    "        \n",
    "        sep_idx = list(i for i, token in enumerate(tokens) if token=='[SEP]')\n",
    "        \n",
    "        token_type_ids = None\n",
    "        \n",
    "        if len(sep_idx) > 1: \n",
    "            token_type_ids = [0]*(sep_idx[0] + 1) + \\\n",
    "            [1]*(sep_idx[1] - sep_idx[0]) + \\\n",
    "            [0]*(self.MAX_SEQ_LEN - sep_idx[1] -1)\n",
    "            \n",
    "        else:\n",
    "            token_type_ids = [0]*(sep_idx[0] + 1) + \\\n",
    "            [0]*(self.MAX_SEQ_LEN - sep_idx[0] -1)\n",
    "            \n",
    "        return token_type_ids\n",
    "    \n",
    "    def _get_attention_mask(self, tokens):\n",
    "        \n",
    "        pad_idx = next(i for i, token in enumerate(tokens) if token == '[PAD]')\n",
    "        attention_mask = [1] * pad_idx + [0] * (self.MAX_SEQ_LEN - pad_idx)\n",
    "        \n",
    "        return attention_mask\n",
    "    \n",
    "    def _preprocess(self):\n",
    "        \n",
    "        self._ids = list()\n",
    "        self._passage_input_ids = list()\n",
    "        self._passage_token_type_ids = list()\n",
    "        self._passage_attention_masks = list()\n",
    "        \n",
    "        self._option_input_ids = list()\n",
    "        self._option_token_type_ids = list()\n",
    "        self._option_attention_masks = list()\n",
    "       \n",
    "        self._answers = list()\n",
    "        \n",
    "        for id_, passage, options, answer in zip(self.ids, self.passages, self.options, self.answers):\n",
    "            \n",
    "            # id\n",
    "            self._ids += [str(id_) + '_' + str(i+1) for i in range(len(options))]\n",
    "            \n",
    "            # passage & options\n",
    "            passage_tokens = self._get_tokens(passage, True)\n",
    "            options_tokens = list(self._get_tokens(option, False) for option in options)\n",
    "            \n",
    "            passage_tokens = self._add_special_token(passage_tokens, True)\n",
    "            options_tokens = list(self._add_special_token(option_tokens, False) for option_tokens in options_tokens)\n",
    "            \n",
    "            passage_input_id = self._get_input_ids(passage_tokens)\n",
    "            options_input_id = list(self._get_input_ids(option_tokens) for option_tokens in options_tokens)\n",
    "            \n",
    "            self._passage_input_ids += [passage_input_id] * len(options)\n",
    "            self._option_input_ids += options_input_id\n",
    "            \n",
    "            passage_token_type_id = self._get_token_type_ids(passage_tokens)\n",
    "            options_token_type_id = list(self._get_token_type_ids(option_tokens) for option_tokens in options_tokens)\n",
    "            \n",
    "            self._passage_token_type_ids += [passage_token_type_id] * len(options)\n",
    "            self._option_token_type_ids += options_token_type_id\n",
    "            \n",
    "            passage_attention_mask = self._get_attention_mask(passage_tokens)\n",
    "            options_attention_mask = list(self._get_attention_mask(option_tokens) for option_tokens in options_tokens)\n",
    "            \n",
    "            self._passage_attention_masks += [passage_attention_mask] * len(options)\n",
    "            self._option_attention_masks += options_attention_mask\n",
    "            \n",
    "            # answer\n",
    "            tmp = [1] * len(options)\n",
    "            tmp[int(answer-1)] = 0\n",
    "            self._answers += tmp\n",
    "            \n",
    "            \n",
    "    def _collate(self, batch, device):\n",
    "        \n",
    "        ids = list()\n",
    "        passage_input_ids = list()\n",
    "        passage_token_type_ids = list()\n",
    "        passage_attention_masks = list()\n",
    "        option_input_ids = list()\n",
    "        option_token_type_ids = list()\n",
    "        option_attention_masks = list()\n",
    "        answers = list()\n",
    "        \n",
    "        for id_, passage_input_id, passage_token_type_id, passage_attention_mask, \\\n",
    "        option_input_id, option_token_type_id, option_attention_mask, answer in batch:\n",
    "            ids.append(id_)\n",
    "            \n",
    "            passage_input_ids.append(torch.LongTensor(passage_input_id).to(device))\n",
    "            passage_token_type_ids.append(torch.LongTensor(passage_token_type_id).to(device))\n",
    "            passage_attention_masks.append(torch.LongTensor(passage_attention_mask).to(device))\n",
    "            \n",
    "            option_input_ids.append(torch.LongTensor(option_input_id).to(device))\n",
    "            option_token_type_ids.append(torch.LongTensor(option_token_type_id).to(device))\n",
    "            option_attention_masks.append(torch.LongTensor(option_attention_mask).to(device))\n",
    "            answers.append(answer)\n",
    "                \n",
    "        passage_input_ids = torch.stack(passage_input_ids, dim=0)\n",
    "        passage_token_type_ids = torch.stack(passage_token_type_ids, dim=0)\n",
    "        passage_attention_masks = torch.stack(passage_attention_masks, dim=0)\n",
    "        \n",
    "        option_input_ids = torch.stack(option_input_ids, dim=0)\n",
    "        option_token_type_ids = torch.stack(option_token_type_ids, dim=0)\n",
    "        option_attention_masks = torch.stack(option_attention_masks, dim=0)\n",
    "        answers = torch.LongTensor(answers).to(device)\n",
    "        \n",
    "        return id_, passage_input_ids, passage_token_type_ids, passage_attention_masks, \\\n",
    "    option_input_ids, option_token_type_ids, option_attention_masks, answers\n",
    "    \n",
    "    def __call__(self, batch_size, do_split, train_ratio, num_workers, device):\n",
    "        \n",
    "        self._preprocess()\n",
    "        \n",
    "        result = None\n",
    "        if do_split:\n",
    "            temp_train, temp_test = tee(torch.bernoulli(train_ratio * torch.ones(len(self.ids))), 2)\n",
    "            temp_train = list(i for i, x in enumerate(temp_train) if x.item() == 1)\n",
    "            temp_test = list(i for i, x in enumerate(temp_test) if x.item() == 0)\n",
    "            \n",
    "            indices_train = list()\n",
    "            for t in temp_train:\n",
    "                indices_train += list(range(5*t, 5*(t+1)))\n",
    "                \n",
    "            indices_test = list()\n",
    "            for t in temp_test:\n",
    "                indices_test += list(range(5*t, 5*(t+1)))\n",
    "            \n",
    "            subset_train = Subset(self, indices_train)\n",
    "            subset_test = Subset(self, indices_test)\n",
    "            \n",
    "            train_loader = DataLoader(subset_train, batch_size=batch_size, num_workers=num_workers, \n",
    "                          collate_fn=partial(self._collate, device=device))\n",
    "            test_loader = DataLoader(subset_test, batch_size=batch_size, num_workers=num_workers, \n",
    "                          collate_fn=partial(self._collate, device=device))\n",
    "            result = (train_loader, test_loader)\n",
    "            \n",
    "        else:\n",
    "            data_loader = DataLoader(self, batch_size=batch_size, num_workers=num_workers, \n",
    "                          collate_fn=partial(self._collate, device=device))\n",
    "            result = data_loader\n",
    "            \n",
    "        return result "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BertWithSan(nn.Module):\n",
    "    def __init__(self, pretrained_model='bert-base-cased', hidden_size=768, class_size=2, dropout_prob=0.4, K=7):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.K = K\n",
    "        \n",
    "        self.bert_p = BertModel.from_pretrained(pretrained_model) # BERT: Passage encoder\n",
    "        self.bert_h = BertModel.from_pretrained(pretrained_model) # BERT: Option encoder\n",
    "        \n",
    "        self.w_1 = nn.Parameter(torch.rand((hidden_size,))) # M^h to alpha\n",
    "        self.W_2 = nn.Parameter(torch.rand((hidden_size, 512))) # M^p to beta \n",
    "        self.W_3 = nn.Parameter(torch.rand((1, 4))) # hidden states to P_r\n",
    "        self.grucell = nn.GRUCell(hidden_size, hidden_size)\n",
    "        \n",
    "        self.dropout = nn.Dropout(dropout_prob) \n",
    "        self.classifier = nn.Linear(hidden_size, 2)\n",
    "        \n",
    "        \n",
    "    def forward(self, passage_input_ids, passage_token_type_ids, passage_attention_masks, \n",
    "                option_input_ids, option_token_type_ids, option_attention_masks):\n",
    "        \n",
    "        # passage input ids shape: (batch_size, max_position_embeddings)\n",
    "        # passage token type ids shape: (batch_size, max_position_embeddings)\n",
    "        # option input ids shape: (batch_size, max_position_embeddings)\n",
    "        # option token type ids shape: (batch_size, max_position_embeddings)\n",
    "        \n",
    "        M_p, _ = self.bert_p(passage_input_ids, passage_token_type_ids, passage_attention_masks, False)\n",
    "        M_h, _ = self.bert_h(option_input_ids, option_token_type_ids, option_attention_masks, False)\n",
    "        # M_p shape: (batch_size, max_position_embeddings, hidden_size)\n",
    "        # M_h shape: (batch_size, max_position_embeddings, hidden_size)\n",
    "        \n",
    "        alpha = F.softmax(self.w_1*M_h).squeeze(2)\n",
    "        # w_1 shape: (1, hidden_size)\n",
    "        # alpha shape: (batch_size, max_position_embeddings)\n",
    "        \n",
    "        s_0 = torch.sum(alpha*M_h, dim=1)\n",
    "        # s_0 shape: (batch_size, hidden_size)\n",
    "        \n",
    "        temp = list()\n",
    "        for k in range(self.K):\n",
    "            \n",
    "            s_k_1 = None\n",
    "            # s^{k-1}\n",
    "            if k == 0:\n",
    "                s_k_1 = s_0\n",
    "            else:\n",
    "                s_k_1 = s_k\n",
    "            \n",
    "            beta = F.softmax((torch.matmul(s_k_1, self.W_2).unsqueeze(1)@M_p).squeeze(1))\n",
    "            # beta shape: (batch_size, hidden_size)\n",
    "            \n",
    "            x_k = torch.sum((beta.unsqueeze(1)*M_p), dim=1)\n",
    "            # x_k shape: (batch_size, hidden_size)\n",
    "            \n",
    "            s_k = self.grucell(x_k, s_k_1)\n",
    "            # s_k shape: (batch_size, hidden_size)\n",
    "            \n",
    "            P_r_k = F.softmax(torch.matmul(self.W_3,\n",
    "                                           torch.stack([s_k, x_k, torch.abs(s_k-x_k), (s_k*x_k)], dim=1)))\n",
    "            # P_r_k shape: (batch_size, 1, hidden_size)\n",
    "            \n",
    "            temp.append(P_r_k)\n",
    "        \n",
    "        P_r = torch.cat(temp, dim=1)\n",
    "        P_r = self.dropout(P_r)\n",
    "        P_r = torch.mean(P_r, dim=1)\n",
    "        # P_r shape: (batch_size, hidden_size)\n",
    "        \n",
    "        logits = self.classifier(P_r)\n",
    "        # logits shape: (batch_size, 2)\n",
    "        \n",
    "        return logits"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### train / evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, data_parallel, data_loader, optimizer, criterion):\n",
    "    \n",
    "    model.train()\n",
    "    if data_parallel: # use Data Parallelism with Multi-GPU\n",
    "        model = nn.DataParallel(model)\n",
    "    epoch_loss = 0\n",
    "    iter_bar = tqdm_notebook(data_loader, desc='Iter (loss=X.XXX)')\n",
    "    \n",
    "    for i, batch in enumerate(iter_bar):\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        passage_input_ids, passage_token_type_ids, passage_attention_masks, \\\n",
    "                option_input_ids, option_token_type_ids, option_attention_masks = batch[1:-1]\n",
    "        label = batch[-1]\n",
    "\n",
    "        logits = model(passage_input_ids, passage_token_type_ids, passage_attention_masks,\n",
    "                           option_input_ids, option_token_type_ids, option_attention_masks)\n",
    "        \n",
    "        loss = criterion(logits, label)\n",
    "        loss.backward()\n",
    "        \n",
    "        optimizer.step()\n",
    "        epoch_loss += loss.item()\n",
    "        iter_bar.set_description('Iter (loss={})'.format(loss.item()))\n",
    "        \n",
    "    return epoch_loss / len(data_loader)\n",
    "\n",
    "\n",
    "def evaluate(model, data_loader, criterion):\n",
    "    \n",
    "    model.eval()\n",
    "    epoch_loss = 0\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        for i, batch in enumerate(data_loader):\n",
    "            \n",
    "            passage_input_ids, passage_token_type_ids, passage_attention_masks, \\\n",
    "                option_input_ids, option_token_type_ids, option_attention_masks = batch[1:-1]\n",
    "            label = batch[-1]\n",
    "            \n",
    "            logits = model(passage_input_ids, passage_token_type_ids, passage_attention_masks,\n",
    "                           option_input_ids, option_token_type_ids, option_attention_masks)\n",
    "            \n",
    "            loss = criterion(logits, label)\n",
    "            epoch_loss += loss.item()\n",
    "            \n",
    "    return epoch_loss / len(data_loader)\n",
    "\n",
    "\n",
    "def score(model, load_path, data_loader, device):\n",
    "    \n",
    "    # load model\n",
    "    if load_path is not None:\n",
    "        model.load_state_dict(torch.load(load_path))\n",
    "        print('Loading the model from', load_path)\n",
    "    \n",
    "    model.eval()\n",
    "    match = 0\n",
    "    N = 0\n",
    "    with torch.no_grad():\n",
    "        for i, batch in enumerate(data_loader):\n",
    "            \n",
    "            passage_input_ids, passage_token_type_ids, passage_attention_masks, \\\n",
    "                option_input_ids, option_token_type_ids, option_attention_masks = batch[1:-1]\n",
    "            label = batch[-1]\n",
    "            \n",
    "            true = torch.argmin(label)\n",
    "            \n",
    "            logits = model(passage_input_ids, passage_token_type_ids, passage_attention_masks,\n",
    "                           option_input_ids, option_token_type_ids, option_attention_masks)\n",
    "            \n",
    "            pred = torch.argmax(logits.transpose(0,1)[0])\n",
    "            \n",
    "            if true.item() == pred.item():\n",
    "                match += 1\n",
    "                \n",
    "            N += 1\n",
    "            \n",
    "    score = match / N\n",
    "    \n",
    "    return score\n",
    "\n",
    "\n",
    "class EarlyStopping:\n",
    "    \"\"\"Early stops the training if validation loss doesn't improve after a given patience.\"\"\"\n",
    "    def __init__(self, save_path, patience=1, verbose=False):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            patience (int): How long to wait agter last time validation loss improved.\n",
    "                            Default: 7\n",
    "            verbose (bool): If True, prints a message for each validation loss improved.\n",
    "                            Default: False\n",
    "        \"\"\"\n",
    "        self.save_path = save_path\n",
    "        self.patience = patience\n",
    "        self.verbose = verbose\n",
    "        self.counter = 0\n",
    "        self.best_score = None\n",
    "        self.early_stop = False\n",
    "        \n",
    "    def __call__(self, val_loss, model):\n",
    "        \n",
    "        score = val_loss\n",
    "        \n",
    "        if self.best_score is None:\n",
    "            torch.save(model.state_dict(), self.save_path)\n",
    "            print(\"Saving the model to\", self.save_path)\n",
    "            self.best_score = score\n",
    "            \n",
    "        elif score > self.best_score:\n",
    "            self.counter += 1 \n",
    "            print('EarlyStopping counter: {} out of {}'.format(self.counter, self.patience))\n",
    "            if self.counter >= self.patience:\n",
    "                self.early_stop = True\n",
    "                \n",
    "        else:\n",
    "            torch.save(model.state_dict(), self.save_path)\n",
    "            print(\"Saving the model to\", self.save_path)\n",
    "            self.best_score = score\n",
    "            self.counter = 0\n",
    "            \n",
    "        return self.early_stop"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### main"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The pre-trained model you are loading is a cased model but you have not set `do_lower_case` to False. We are setting `do_lower_case=False` for you but you may want to check this behavior.\n"
     ]
    }
   ],
   "source": [
    "data_loading_train = SANDataLoading.from_tsv('KSAT_TRAIN.tsv')\n",
    "train_loader = data_loading_train(batch_size=5, do_split=False, train_ratio=0.7, num_workers=0, device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The pre-trained model you are loading is a cased model but you have not set `do_lower_case` to False. We are setting `do_lower_case=False` for you but you may want to check this behavior.\n"
     ]
    }
   ],
   "source": [
    "data_loading_test = SANDataLoading.from_tsv('KSAT_TEST.tsv')\n",
    "test_loader = data_loading_test(batch_size=5, do_split=False, train_ratio=0.7, num_workers=0, device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = BertWithSan().to(device)\n",
    "optimizer = optim.Adam(model.parameters(), lr=5e-5)\n",
    "criterion = nn.CrossEntropyLoss(torch.FloatTensor([5,1.25]).to(device))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "now = datetime.now()\n",
    "year = str(now.year)[-2:]\n",
    "month = '{00}'.format(now.month)\n",
    "day = '{00}'.format(now.day)\n",
    "hour = '{00}'.format(now.hour)\n",
    "min_ = '{00}'.format(now.minute)\n",
    "\n",
    "model_name = model.__class__.__name__ +'_'+year+month+day+hour+min_\n",
    "# PATH = os.path.join('./',model_name+'_'epoch+'.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5a6f47093e7944b697e15ae18d455e93",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/lilydpjja/.local/lib/python3.5/site-packages/ipykernel_launcher.py:32: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "/home/lilydpjja/.local/lib/python3.5/site-packages/ipykernel_launcher.py:49: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "/home/lilydpjja/.local/lib/python3.5/site-packages/ipykernel_launcher.py:59: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Saving the model to ./BertWithSan_196271656_0.pt\n",
      "Epoch 1| Train Loss : 0.6746212026515565\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1efd34aae8a54c47b5a888777a1659c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 2| Train Loss : 0.5951903940795304\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1fb72123e6fa4f5a8971a8ce1a50b082",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 3| Train Loss : 0.4711210793682507\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e3435ffd982f437b9602946e0abf08d1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 4| Train Loss : 0.5138380392998844\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "db522fa8995c4ab881dfa55af3767a77",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 5| Train Loss : 0.42781256856275846\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "96396ec7c31d4d9f9eba64a7a3cd4c40",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Saving the model to ./BertWithSan_196271656_5.pt\n",
      "Epoch 6| Train Loss : 0.6075884458887113\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1073cca78eaf4e579169396a446783a6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 7| Train Loss : 0.46222117078768743\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "583344a4ffae4d63b53bcc2b64046917",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 8| Train Loss : 0.637736383390117\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "107e4157c6ba466c899b6d54ad8b51ee",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 9| Train Loss : 0.7025562526343705\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "96211476f80a435cb10f91449c69ffd4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 10| Train Loss : 0.6971912843066377\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c08ebb4cff6f46b2a88c0d5de72c2a8d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Saving the model to ./BertWithSan_196271656_10.pt\n",
      "Epoch 11| Train Loss : 0.6961713261805571\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1472a4a7a24645ffae41aad4e6da8d38",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 12| Train Loss : 0.6819497602713572\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "691f7a97db4541c28ee05ac803f03dbe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 13| Train Loss : 0.6951787794178182\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e4f6bd1ea9ae496b815efedba60c6c29",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 14| Train Loss : 0.6978334571634021\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7d0fbf1dd60f4b3b8795f636a22f7952",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 15| Train Loss : 0.6983771541675964\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9787351722594dd7a2f6c3dd3d3a3f9e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Saving the model to ./BertWithSan_196271656_15.pt\n",
      "Epoch 16| Train Loss : 0.7015916913360745\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "813afc60acb44fdbbb921c7f8925fe8d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 17| Train Loss : 0.6953795683074308\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e165d6e1a61547208381f5d03fe708be",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 18| Train Loss : 0.696494239646119\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dfaf4df71fc14bcfb6c008ad8a00fd23",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 19| Train Loss : 0.6964538423271922\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "005480be828548f5a9a657f90db47833",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 20| Train Loss : 0.6949351021221706\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dd34dae6ba924cc98414e71bde1c41f6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Saving the model to ./BertWithSan_196271656_20.pt\n",
      "Epoch 21| Train Loss : 0.6939907513655625\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d01992b61dd54042ac3383f06427290e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 22| Train Loss : 0.6936609835593731\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e23763ed3705416aa7f96e77fa9dd867",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 23| Train Loss : 0.694457460301263\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1d3959f80b0548219eb6733cdef419d2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 24| Train Loss : 0.6943544382398779\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b49748fc9da64f35a7306f9e12f737e2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 25| Train Loss : 0.6929478311693513\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "43a7d03f50724e5faa67e2740dc31c80",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Saving the model to ./BertWithSan_196271656_25.pt\n",
      "Epoch 26| Train Loss : 0.6937886233453626\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "41dfa6c7e32a4ae9ac5f05bd18bcc63f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 27| Train Loss : 0.6945131372321736\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "12dd85d3beaa4cea84fbce68d7080b55",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 28| Train Loss : 0.69403442252766\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a337dd54a9ce400ebb86c237f53107ee",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 29| Train Loss : 0.6954437900673259\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "24d05ff4d5fb45349e60535f128db4bd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 30| Train Loss : 0.6949483992217423\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d9c8d35969f14bf68e5b0b44549d7f35",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Saving the model to ./BertWithSan_196271656_30.pt\n",
      "Epoch 31| Train Loss : 0.6946029259786978\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8297dcccf1d74b068d145184acc93a4e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 32| Train Loss : 0.694708419775034\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "53e57c840fc146f684175051ca34b351",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 33| Train Loss : 0.6911649958653884\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a8f8c1ebc2744ea2b4217afd8dd22311",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 34| Train Loss : 0.6989743505979513\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "60777828d9f940aeb4da24f06074adb2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 35| Train Loss : 0.6915545255332798\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dab549d472094944a74754c392c4cbce",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Saving the model to ./BertWithSan_196271656_35.pt\n",
      "Epoch 36| Train Loss : 0.6964405599352601\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0622cbed3f9a476d9b79f3ecf5e7ac9b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 37| Train Loss : 0.6927487308328802\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "814ca6e63165439486ab048b5214768a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 38| Train Loss : 0.693624522159626\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "787093869357406cbbceb33bd7a6d030",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 39| Train Loss : 0.694505946744572\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5e1270c09d4446aa93c0a1b7c2d275e3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 40| Train Loss : 0.6968522061001171\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f55db587883e471887bb48d5086b393e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Saving the model to ./BertWithSan_196271656_40.pt\n",
      "Epoch 41| Train Loss : 0.6941272448409688\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4e926ecd676741adac60f1ca8d6d44b7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 42| Train Loss : 0.6954089463531197\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cd0ddf896e7f4687bfc43b6112fad2ff",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 43| Train Loss : 0.6948294290474483\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6e796a8d16f442d0b688cc1147f01418",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 44| Train Loss : 0.6932845372658272\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1bb55ee25cd24862b5fbe8861ac4e4e8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 45| Train Loss : 0.6937738368263492\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "048aa17dd5334b07a92bcdbddb06cb0f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Saving the model to ./BertWithSan_196271656_45.pt\n",
      "Epoch 46| Train Loss : 0.6933478099959237\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bb5f0a9b83dd43cca872a5d60379a8a1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 47| Train Loss : 0.6947168497296122\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "69ad20fe00204b65b5e3c0eac5d636ef",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 48| Train Loss : 0.6939676208929582\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f476457012da49f5a717a39fd31332d9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 49| Train Loss : 0.6934667763771949\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c7f11b4be3404be98e9798022a21b6ac",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iter (loss=X.XXX)', max=770, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 50| Train Loss : 0.6956129141442188\n",
      "Saving the model to ./BertWithSan_196271656.pt\n"
     ]
    }
   ],
   "source": [
    "N_EPOCHS = 50\n",
    "\n",
    "# SAVE_PATH = PATH\n",
    "# print('SAVE PATH: {}'.format(SAVE_PATH))\n",
    "\n",
    "patience = 50\n",
    "verbose = True\n",
    "\n",
    "# early_stopping = EarlyStopping(SAVE_PATH, patience, verbose)\n",
    "\n",
    "for epoch in range(N_EPOCHS):\n",
    "    \n",
    "    train_loss = train(model, False, train_loader, optimizer, criterion)\n",
    "#     test_loss = evaluate(model, test_loader, criterion)\n",
    "    \n",
    "#     early_stop = early_stopping(test_loss, model)\n",
    "#     if early_stop:\n",
    "#         print('Epoch {}| Train Loss : {}, Test Loss : {}'.format(epoch+1, train_loss, test_loss))\n",
    "#         break\n",
    "    if epoch % 5 == 0:\n",
    "        torch.save(model.state_dict(), os.path.join('./',model_name+'_'+str(epoch)+'.pt'))\n",
    "        print(\"Saving the model to\", os.path.join('./',model_name+'_'+str(epoch)+'.pt'))\n",
    "    print('Epoch {}| Train Loss : {}'.format(epoch+1, train_loss))\n",
    "    \n",
    "torch.save(model.state_dict(), os.path.join('./',model_name+'.pt'))\n",
    "print(\"Saving the model to\", os.path.join('./',model_name+'.pt'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/lilydpjja/.local/lib/python3.5/site-packages/ipykernel_launcher.py:32: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "/home/lilydpjja/.local/lib/python3.5/site-packages/ipykernel_launcher.py:49: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "/home/lilydpjja/.local/lib/python3.5/site-packages/ipykernel_launcher.py:59: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train | 0.19220779220779222\n",
      "Test | 0.06315789473684211\n"
     ]
    }
   ],
   "source": [
    "# LOAD_PATH = PATH\n",
    "\n",
    "train_score = score(model, None, train_loader, device)\n",
    "test_score = score(model, None, test_loader, device)\n",
    "print('Train | {}\\nTest | {}'.format(train_score, test_score))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Final models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FileName: BertWithSan_196271656.pt\n",
      "Loading the model from ./BertWithSan_196271656.pt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/lilydpjja/.local/lib/python3.5/site-packages/ipykernel_launcher.py:32: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "/home/lilydpjja/.local/lib/python3.5/site-packages/ipykernel_launcher.py:49: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "/home/lilydpjja/.local/lib/python3.5/site-packages/ipykernel_launcher.py:59: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading the model from ./BertWithSan_196271656.pt\n",
      "Train | 0.19220779220779222\n",
      "Test | 0.06315789473684211\n",
      "FileName: BertWithSan_196271656_0.pt\n",
      "Loading the model from ./BertWithSan_196271656_0.pt\n",
      "Loading the model from ./BertWithSan_196271656_0.pt\n",
      "Train | 0.34675324675324676\n",
      "Test | 0.35789473684210527\n",
      "FileName: BertWithSan_196271656_10.pt\n",
      "Loading the model from ./BertWithSan_196271656_10.pt\n",
      "Loading the model from ./BertWithSan_196271656_10.pt\n",
      "Train | 0.6948051948051948\n",
      "Test | 0.29473684210526313\n",
      "FileName: BertWithSan_196271656_15.pt\n",
      "Loading the model from ./BertWithSan_196271656_15.pt\n",
      "Loading the model from ./BertWithSan_196271656_15.pt\n",
      "Train | 0.6441558441558441\n",
      "Test | 0.3684210526315789\n",
      "FileName: BertWithSan_196271656_20.pt\n",
      "Loading the model from ./BertWithSan_196271656_20.pt\n",
      "Loading the model from ./BertWithSan_196271656_20.pt\n",
      "Train | 0.44935064935064933\n",
      "Test | 0.17894736842105263\n",
      "FileName: BertWithSan_196271656_25.pt\n",
      "Loading the model from ./BertWithSan_196271656_25.pt\n",
      "Loading the model from ./BertWithSan_196271656_25.pt\n",
      "Train | 0.18051948051948052\n",
      "Test | 0.21052631578947367\n",
      "FileName: BertWithSan_196271656_30.pt\n",
      "Loading the model from ./BertWithSan_196271656_30.pt\n",
      "Loading the model from ./BertWithSan_196271656_30.pt\n",
      "Train | 0.17142857142857143\n",
      "Test | 0.10526315789473684\n",
      "FileName: BertWithSan_196271656_35.pt\n",
      "Loading the model from ./BertWithSan_196271656_35.pt\n",
      "Loading the model from ./BertWithSan_196271656_35.pt\n",
      "Train | 0.18701298701298702\n",
      "Test | 0.10526315789473684\n",
      "FileName: BertWithSan_196271656_40.pt\n",
      "Loading the model from ./BertWithSan_196271656_40.pt\n",
      "Loading the model from ./BertWithSan_196271656_40.pt\n",
      "Train | 0.1974025974025974\n",
      "Test | 0.25263157894736843\n",
      "FileName: BertWithSan_196271656_45.pt\n",
      "Loading the model from ./BertWithSan_196271656_45.pt\n",
      "Loading the model from ./BertWithSan_196271656_45.pt\n",
      "Train | 0.1961038961038961\n",
      "Test | 0.21052631578947367\n",
      "FileName: BertWithSan_196271656_5.pt\n",
      "Loading the model from ./BertWithSan_196271656_5.pt\n",
      "Loading the model from ./BertWithSan_196271656_5.pt\n",
      "Train | 0.5727272727272728\n",
      "Test | 0.30526315789473685\n"
     ]
    }
   ],
   "source": [
    "for file in sorted(filter(lambda x: x.startswith('BertWithSan_196271656'), os.listdir('./'))):\n",
    "    path = os.path.join('./', file)\n",
    "    print('FileName: {}'.format(file))\n",
    "    train_score = score(model, path, train_loader, device)\n",
    "    test_score = score(model, path, test_loader, device)\n",
    "    print('Train | {}\\nTest | {}'.format(train_score, test_score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
